{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "strange-cartoon",
   "metadata": {},
   "source": [
    "# TimeEval result analysis\n",
    "\n",
    "Reads the results from a TimeEval run and compiles a small report. Change the constants and the configuration to compile the report for another TimeEval run."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "elegant-fellow",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Automatically reload packages:\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "directed-instruction",
   "metadata": {},
   "outputs": [],
   "source": [
    "# imports\n",
    "import warnings\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import scipy as sp\n",
    "import plotly.offline as py\n",
    "import plotly.graph_objects as go\n",
    "import plotly.figure_factory as ff\n",
    "import plotly.express as px\n",
    "from plotly.subplots import make_subplots\n",
    "from pathlib import Path\n",
    "from timeeval import Datasets"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "synthetic-motivation",
   "metadata": {},
   "source": [
    "## Configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "textile-preview",
   "metadata": {},
   "outputs": [],
   "source": [
    "# constants and configuration\n",
    "data_path = Path(\"../data\") / \"test-cases\"\n",
    "result_root_path = Path(\"../results\")\n",
    "result_paths = [d for d in result_root_path.iterdir() if d.is_dir()]\n",
    "result_paths"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "surface-tanzania",
   "metadata": {},
   "source": [
    "Select a results folder:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "spiritual-emergency",
   "metadata": {},
   "outputs": [],
   "source": [
    "result_path = result_root_path / \"2021-09-22_default-params-1&2&3-merged\"\n",
    "#result_path = result_paths[-1]\n",
    "\n",
    "# load results\n",
    "print(f\"Reading results from {result_path.resolve()}\")\n",
    "\n",
    "df = pd.read_csv(result_path / \"results.csv\")\n",
    "df[\"dataset_name\"] = df[\"dataset\"].str.split(\".\").str[0]\n",
    "\n",
    "def load_scores_df(algorithm_name, dataset_id, repetition=1):\n",
    "    params_id = df.loc[(df[\"algorithm\"] == algorithm_name) & (df[\"collection\"] == dataset_id[0]) & (df[\"dataset\"] == dataset_id[1]), \"hyper_params_id\"].item()\n",
    "    path = (\n",
    "        result_path /\n",
    "        algorithm_name /\n",
    "        params_id /\n",
    "        dataset_id[0] /\n",
    "        dataset_id[1] /\n",
    "        str(repetition) /\n",
    "        \"anomaly_scores.ts\"\n",
    "    )\n",
    "    return pd.read_csv(path, header=None)\n",
    "\n",
    "# load dataset metadata\n",
    "dmgr = Datasets(data_path)\n",
    "\n",
    "def plot_scores(algorithm_name, dataset_name):\n",
    "    if isinstance(algorithm_name, str):\n",
    "        algorithms = [algorithm_name]\n",
    "    else:\n",
    "        algorithms = algorithm_name\n",
    "    # construct dataset ID\n",
    "    dataset_id = (\"GutenTAG\", f\"{dataset_name}.unsupervised\")\n",
    "\n",
    "    # load dataset details\n",
    "    df_dataset = dmgr.get_dataset_df(dataset_id)\n",
    "\n",
    "    # check if dataset is multivariate\n",
    "    dataset_dim = df.loc[df[\"dataset_name\"] == dataset_name, \"dataset_input_dimensionality\"].unique().item()\n",
    "    dataset_dim = dataset_dim.lower()\n",
    "    \n",
    "    auroc = {}\n",
    "    df_scores = pd.DataFrame(index=df_dataset.index)\n",
    "    skip_algos = []\n",
    "    for algo in algorithms:\n",
    "        # get algorithm metric results\n",
    "        try:\n",
    "            auroc[algo] = df.loc[(df[\"algorithm\"] == algo) & (df[\"dataset_name\"] == dataset_name), \"ROC_AUC\"].item()\n",
    "        except ValueError:\n",
    "            warnings.warn(f\"No scores found! Probably {algo} was not executed on {dataset_name}.\")\n",
    "            auroc[algo] = -1\n",
    "            skip_algos.append(algo)\n",
    "            continue\n",
    "\n",
    "        # load scores\n",
    "        training_type = df.loc[df[\"algorithm\"] == algo, \"algo_training_type\"].values[0].lower().replace(\"_\", \"-\")\n",
    "        try:\n",
    "            df_scores[algo] = load_scores_df(algo, (\"GutenTAG\", f\"{dataset_name}.{training_type}\")).iloc[:, 0]\n",
    "        except (ValueError, FileNotFoundError):\n",
    "            warnings.warn(f\"No scores found! Probably {algo} was not executed on {dataset_name}.\")\n",
    "            df_scores[algo] = np.nan\n",
    "            skip_algos.append(algo)\n",
    "    algorithms = [a for a in algorithms if a not in skip_algos]\n",
    "\n",
    "    # Create plot\n",
    "    fig = make_subplots(2, 1)\n",
    "    if dataset_dim == \"multivariate\":\n",
    "        for i in range(1, df_dataset.shape[1]-1):\n",
    "            fig.add_trace(go.Scatter(x=df_dataset.index, y=df_dataset.iloc[:, i], name=f\"channel-{i}\"), 1, 1)\n",
    "    else:\n",
    "        fig.add_trace(go.Scatter(x=df_dataset.index, y=df_dataset.iloc[:, 1], name=\"timeseries\"), 1, 1)\n",
    "    fig.add_trace(go.Scatter(x=df_dataset.index, y=df_dataset[\"is_anomaly\"], name=\"label\"), 2, 1)\n",
    "    for algo in algorithms:\n",
    "        fig.add_trace(go.Scatter(x=df_scores.index, y=df_scores[algo], name=f\"{algo}={auroc[algo]:.4f}\"), 2, 1)\n",
    "    fig.update_xaxes(matches=\"x\")\n",
    "    fig.update_layout(\n",
    "        title=f\"Results of {','.join(algorithms)} on {dataset_name}\",\n",
    "        height=400\n",
    "    )\n",
    "    return py.iplot(fig)\n",
    "\n",
    "def plot_datasets(datasets, max_channels = 20):\n",
    "    if isinstance(datasets, str):\n",
    "        datasets = [datasets]\n",
    "    else:\n",
    "        datasets = datasets\n",
    "    n_datasets = len(datasets)\n",
    "    \n",
    "    # Create plot\n",
    "    fig = make_subplots(n_datasets, 1)\n",
    "    for i, d in enumerate(datasets):\n",
    "        # construct dataset ID\n",
    "        dataset_id = (\"GutenTAG\", f\"{d}.unsupervised\")\n",
    "        \n",
    "        # load dataset details\n",
    "        try:\n",
    "            df_dataset = dmgr.get_dataset_df(dataset_id)\n",
    "        except Exception as e:\n",
    "            warnings.warn(f\"Could not load dataset {d}, because {repr(e)}\")\n",
    "            continue\n",
    "\n",
    "        # get algorithm metric results\n",
    "        try:\n",
    "            auroc = df.loc[df[\"dataset_name\"] == d, \"ROC_AUC\"].median()\n",
    "        except ValueError:\n",
    "            warnings.warn(f\"No scores found for dataset {d} found!\")\n",
    "            auroc = -1\n",
    "            continue\n",
    "\n",
    "        for j in range(1, min(df_dataset.shape[1]-1, max_channels+1)):\n",
    "            fig.add_trace(go.Scatter(\n",
    "                x=df_dataset.index,\n",
    "                y=df_dataset.iloc[:, j],\n",
    "                name=f\"{d} channel {j} ({auroc:.4f})\",\n",
    "            ), i+1, 1)\n",
    "\n",
    "        # mark anomaly regions\n",
    "        s = df_dataset[\"is_anomaly\"].diff()\n",
    "        anomaly_regions = zip(s[s== 1].index, s[s == -1].index)\n",
    "        for s, e in anomaly_regions:\n",
    "            fig.add_vrect(x0=s-1, x1=e,\n",
    "                          exclude_empty_subplots=True,\n",
    "                          line_width=0,\n",
    "                          fillcolor=\"red\",\n",
    "                          opacity=0.3,\n",
    "                          annotation_text=\"anomaly\",\n",
    "                          annotation_position=\"top left\",\n",
    "                          row=i+1,\n",
    "                          col=1)\n",
    "\n",
    "#     fig.update_xaxes(matches=\"x\")\n",
    "    fig.update_layout(\n",
    "        title=f\"Datasets and ground truth of {','.join(datasets)} datasets\",\n",
    "        height=200*n_datasets if n_datasets > 1 else 400\n",
    "    )\n",
    "    return py.iplot(fig)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "streaming-board",
   "metadata": {},
   "source": [
    "Only consider the best run for each `algorithm`-`dataset`-combination (over all `hyper_params` and `repetition`s) for the analysis in this notebook:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "single-agent",
   "metadata": {},
   "outputs": [],
   "source": [
    "def filter_groups(group):\n",
    "    if len(group) > 1:\n",
    "        group = group.sort_values(by=\"ROC_AUC\", ascending=False)\n",
    "    return group[:1]\n",
    "\n",
    "df_grouped = df.groupby(by=[\"algorithm\", \"collection\", \"dataset\"])\n",
    "df_grouped = df_grouped.apply(filter_groups)\n",
    "df_grouped.reset_index(drop=True, inplace=True)\n",
    "df = df_grouped\n",
    "df = df.sort_values(by=[\"algorithm\", \"dataset\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "julian-produce",
   "metadata": {},
   "source": [
    "## Analyze TimeEval results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "disturbed-impact",
   "metadata": {},
   "outputs": [],
   "source": [
    "df[[\"algorithm\", \"dataset_name\", \"status\", \"AVERAGE_PRECISION\", \"PR_AUC\", \"RANGE_PR_AUC\", \"ROC_AUC\", \"execute_main_time\", \"hyper_params\"]]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "casual-rubber",
   "metadata": {},
   "source": [
    "### Errors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "incident-clarity",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "df_error_counts = df.pivot_table(index=[\"algo_training_type\", \"algorithm\"], columns=[\"status\"], values=\"repetition\", aggfunc=\"count\")\n",
    "df_error_counts = df_error_counts.fillna(value=0).astype(np.int64)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "signed-recycling",
   "metadata": {},
   "source": [
    "#### Aggregation of errors per algorithm grouped by algorithm training type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "sharing-lewis",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "for tpe in [\"SEMI_SUPERVISED\", \"SUPERVISED\", \"UNSUPERVISED\"]:\n",
    "    print(tpe)\n",
    "    py.iplot(ff.create_table(df_error_counts.loc[tpe], index=True))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "quantitative-wednesday",
   "metadata": {},
   "source": [
    "#### Slow algorithms\n",
    "\n",
    "Algorithms, for which more than 50% of all executions ran into the timeout."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bigger-africa",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_error_counts[df_error_counts[\"Status.TIMEOUT\"] > (df_error_counts[\"Status.ERROR\"] + df_error_counts[\"Status.OK\"])]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "shaped-outside",
   "metadata": {},
   "source": [
    "#### Broken algorithms\n",
    "\n",
    "Algorithms, which failed for at least 50% of the executions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "optical-elder",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "error_threshold = 0.5\n",
    "df_error_counts[df_error_counts[\"Status.ERROR\"] > error_threshold*(\n",
    "    df_error_counts[\"Status.TIMEOUT\"] + df_error_counts[\"Status.ERROR\"] + df_error_counts[\"Status.OK\"]\n",
    ")]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "educated-observer",
   "metadata": {},
   "source": [
    "### Algorithm quality assessment"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "native-connectivity",
   "metadata": {},
   "source": [
    "#### Overall algorithm performance based on ROC_AUC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "indonesian-button",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "aggregations = [\"min\", \"mean\", \"median\", \"max\"]\n",
    "df_overall_scores = df.pivot_table(index=\"algorithm\", values=\"ROC_AUC\", aggfunc=aggregations)\n",
    "df_overall_scores.columns = aggregations\n",
    "df_overall_scores = df_overall_scores.sort_values(by=\"median\", ascending=False)\n",
    "\n",
    "df_overall_scores.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "neutral-princeton",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_asl = df.pivot(index=\"algorithm\", columns=\"dataset_name\", values=\"ROC_AUC\")\n",
    "df_asl = df_asl.dropna(axis=0, how=\"all\").dropna(axis=1, how=\"all\")\n",
    "df_asl[\"median\"] = df_asl.median(axis=1)\n",
    "df_asl = df_asl.sort_values(by=\"median\", ascending=True)\n",
    "df_asl = df_asl.drop(columns=\"median\").T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "national-attribute",
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = go.Figure()\n",
    "for c in df_asl.columns:\n",
    "    fig.add_trace(go.Violin(\n",
    "        y=df_asl[c],\n",
    "        name=c\n",
    "    ))\n",
    "fig.update_traces(meanline_visible=True, box_visible=True)\n",
    "fig.update_layout(\n",
    "    title={\"text\":\"AUC_ROC violin plots\", \"xanchor\": \"center\", \"x\": 0.5},\n",
    "    yaxis_title=\"AUC_ROC score\",\n",
    "    legend_title=\"Algorithms\",\n",
    "    violingap=0\n",
    ")\n",
    "py.iplot(fig)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "popular-question",
   "metadata": {},
   "outputs": [],
   "source": [
    "n_show = 10\n",
    "n_show = n_show // 2\n",
    "fig = go.Figure()\n",
    "for i, c in enumerate(df_asl.columns):\n",
    "    fig.add_trace(go.Box(\n",
    "        x=df_asl[c],\n",
    "        name=c,\n",
    "        boxpoints=False,\n",
    "        visible=None if i < n_show or i > len(df_asl.columns)-n_show-1 else \"legendonly\"\n",
    "    ))\n",
    "fig.update_layout(\n",
    "    title={\"text\":\"AUC_ROC box plots\", \"xanchor\": \"center\", \"x\": 0.5},\n",
    "    xaxis_title=\"AUC_ROC score\",\n",
    "    legend_title=\"Algorithms\"\n",
    ")\n",
    "py.iplot(fig)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "satellite-factor",
   "metadata": {},
   "source": [
    "#### Scores of best algorithms\n",
    "\n",
    "Please select a dataset (and algorithm if needed):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "interracial-weekend",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "dataset_name = \"sinus-combined-diff-2\"\n",
    "algorithm_name = None\n",
    "\n",
    "plot_scores(algorithm_name if algorithm_name else df_asl.columns[-4:], dataset_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "hawaiian-trust",
   "metadata": {},
   "source": [
    "### Dataset inspection"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "497eeb1b",
   "metadata": {},
   "source": [
    "#### Datasets based on the achieved AUC_ROC scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "talented-cowboy",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_datasets = df.pivot(index=\"dataset_name\", columns=\"algorithm\", values=\"ROC_AUC\")\n",
    "#df_datasets = df_datasets.dropna(axis=0, how=\"all\").dropna(axis=1, how=\"all\")\n",
    "df_datasets[\"median\"] = df_datasets.median(axis=1)\n",
    "df_datasets = df_datasets.sort_values(by=\"median\", ascending=True)\n",
    "df_datasets = df_datasets.drop(columns=\"median\").T\n",
    "\n",
    "def plot_dataset_boxplot(characteristic):\n",
    "    df_c = df_datasets.drop(columns=[c for c in df_datasets.columns if characteristic != c.split(\"-\")[1]])\n",
    "    fig = go.Figure()\n",
    "    for i, c in enumerate(df_c.columns):\n",
    "        base_osci = c.split(\"-\")[0]\n",
    "        fig.add_trace(go.Box(\n",
    "            x=df_c[c],\n",
    "            name=c,\n",
    "            boxpoints=False,\n",
    "            legendgroup=base_osci,\n",
    "            visible=\"legendonly\" if base_osci != \"sinus\" else None\n",
    "        ))\n",
    "    fig.update_layout(\n",
    "        title={\"text\": f\"Dataset scores by characteristic '{characteristic}'\", \"xanchor\": \"center\", \"x\": 0.5},\n",
    "        xaxis_title=\"AUC_ROC score\",\n",
    "        legend_title=\"Datasets\"\n",
    "    )\n",
    "    return py.iplot(fig)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "abstract-niagara",
   "metadata": {},
   "outputs": [],
   "source": [
    "n_show = 10\n",
    "n_show = n_show // 2\n",
    "fig = go.Figure()\n",
    "for i, c in enumerate(df_datasets.columns):\n",
    "    fig.add_trace(go.Box(\n",
    "        x=df_datasets[c],\n",
    "        name=c,\n",
    "        boxpoints=False,\n",
    "        visible=None if i < n_show or i > len(df_datasets.columns)-n_show-1 else \"legendonly\"\n",
    "    ))\n",
    "fig.update_layout(\n",
    "    title={\"text\":\"AUC_ROC box plots\", \"xanchor\": \"center\", \"x\": 0.5},\n",
    "    xaxis_title=\"AUC_ROC score\",\n",
    "    legend_title=\"Datasets\"\n",
    ")\n",
    "py.iplot(fig)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c7025f1b",
   "metadata": {},
   "source": [
    "Easiest (best performing) datasets:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fd5dce0c",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_datasets(df_datasets.columns[-4:][::-1])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "90c7dee6",
   "metadata": {},
   "source": [
    "Hardest (lowest performing) datasets:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "associate-forwarding",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_datasets(df_datasets.columns[:4])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fatal-framing",
   "metadata": {},
   "source": [
    "#### Scores depending on anomaly position"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "potential-quarterly",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_dataset_boxplot(\"position\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fifteen-result",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_datasets([\"sinus-position-beginning\", \"sinus-position-middle\", \"sinus-position-end\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "acb1381e",
   "metadata": {},
   "source": [
    "#### Scores depending on anomaly characteristics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e44e0e2e",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_dataset_boxplot(\"noise\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f06b28db",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_dataset_boxplot(\"length\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4fc520bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_dataset_boxplot(\"type\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9a3b1ca0",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_datasets([\"sinus-type-amplitude\", \"sinus-type-trend\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c6ac91a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_dataset_boxplot(\"trend\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9033ce22",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_dataset_boxplot(\"channels\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "excessive-costa",
   "metadata": {},
   "source": [
    "## Baselines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "capable-commonwealth",
   "metadata": {},
   "outputs": [],
   "source": [
    "baselines = [\"normal\", \"increasing\", \"Random\"]\n",
    "df_baselines = df[df[\"algorithm\"].isin(baselines)]\n",
    "print(\"Mean scores of the baselines algorithms\")\n",
    "df_baselines.groupby(by=\"algorithm\").mean()[[\"AVERAGE_PRECISION\", \"PR_AUC\", \"RANGE_PR_AUC\", \"ROC_AUC\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "amber-terrorist",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_base = df_baselines.pivot(index=\"algorithm\", columns=\"dataset_name\", values=\"ROC_AUC\")\n",
    "df_base = df_base.dropna(axis=0, how=\"all\").dropna(axis=1, how=\"all\")\n",
    "df_base[\"median\"] = df_base.median(axis=1)\n",
    "df_base = df_base.sort_values(by=\"median\", ascending=True)\n",
    "df_base = df_base.drop(columns=\"median\").T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "conceptual-policy",
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = go.Figure()\n",
    "for c in df_base.columns:\n",
    "    fig.add_trace(go.Violin(\n",
    "        y=df_base[c],\n",
    "        name=c\n",
    "    ))\n",
    "fig.update_traces(meanline_visible=True, box_visible=True)\n",
    "fig.update_layout(\n",
    "    title={\"text\":\"AUC_ROC for the baseline algorithms\", \"xanchor\": \"center\", \"x\": 0.5},\n",
    "    yaxis_title=\"AUC_ROC score\",\n",
    "    legend_title=\"Baseline algorithms\",\n",
    "    violingap=0\n",
    ")\n",
    "py.iplot(fig)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "mediterranean-consumption",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_scores(baselines, \"sinus-diff-count-5\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "practical-bottom",
   "metadata": {},
   "source": [
    "## Experimentation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "copyrighted-sleep",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_scores(df_asl.columns[-10:], \"rw-channels-single-of-5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "wicked-import",
   "metadata": {},
   "outputs": [],
   "source": [
    "n_show = 10\n",
    "n_show = n_show // 2\n",
    "fig = go.Figure()\n",
    "for i, c in enumerate(df_asl.columns):\n",
    "    fig.add_trace(go.Box(\n",
    "        x=df_asl[c],\n",
    "        name=c,\n",
    "        boxpoints=False,\n",
    "        visible=\"legendonly\" if n_show < i < len(df_asl.columns)-n_show else None\n",
    "    ))\n",
    "fig.update_layout(\n",
    "    title={\"text\":\"AUC_ROC box plots\", \"xanchor\": \"center\", \"x\": 0.5},\n",
    "    xaxis_title=\"AUC_ROC score\",\n",
    "    legend_title=\"Algorithms\"\n",
    ")\n",
    "py.iplot(fig)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "greek-postage",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_scores(\"Hybrid-KNN\", \"ecg-diff-count-4\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "protective-gravity",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_path(algorithm_name, dataset_id, repetition=1):\n",
    "    params_id = df.loc[(df[\"algorithm\"] == algorithm_name) & (df[\"collection\"] == dataset_id[0]) & (df[\"dataset\"] == dataset_id[1]), \"hyper_params_id\"].item()\n",
    "    path = (\n",
    "        result_path /\n",
    "        algorithm_name /\n",
    "        params_id /\n",
    "        dataset_id[0] /\n",
    "        dataset_id[1] /\n",
    "        str(repetition)\n",
    "    )\n",
    "    return path\n",
    "dd = pd.read_csv(get_path(\"DBStream\", (\"GutenTAG\", \"ecg-diff-count-6.unsupervised\")) / \"docker-algorithm-scores.csv\", header=None)\n",
    "fig = go.Figure()\n",
    "fig.add_trace(go.Scatter(\n",
    "    x=dd.index,\n",
    "    y=dd.iloc[:, 0],\n",
    "    name=\"scores\",\n",
    "))\n",
    "fig.update_layout(\n",
    "    title={\"text\":\"DBStream original scores\", \"xanchor\": \"center\", \"x\": 0.5},\n",
    "    xaxis_title=\"AUC_ROC score\",\n",
    "    legend_title=\"Algorithms\"\n",
    ")\n",
    "py.iplot(fig)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bright-noise",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_tmp = df.copy()\n",
    "df_tmp[\"overall_time\"] = df[\"execute_main_time\"].fillna(0) + df[\"train_main_time\"].fillna(0)\n",
    "df_tmp = df_tmp.pivot_table(index=\"algorithm\", values=[\"ROC_AUC\", \"overall_time\"], aggfunc=\"median\")\n",
    "\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "df_tmp[\"overall_time\"] = 1 - MinMaxScaler(feature_range=(1e-6, 1)).fit_transform(df_tmp[\"overall_time\"].values.reshape(-1, 1)).reshape(-1)\n",
    "df_tmp = df_tmp.replace(0, np.nan)\n",
    "df_tmp[\"weighted ROC_AUC\"] = df_tmp[\"ROC_AUC\"] * df_tmp[\"overall_time\"]\n",
    "\n",
    "df_tmp = df_tmp.sort_values(by=\"weighted ROC_AUC\", ascending=True, na_position=\"first\")\n",
    "\n",
    "df_tmp.reset_index(drop=False, inplace=True)\n",
    "fig = px.bar(df_tmp, x=\"algorithm\", y=[\"ROC_AUC\", \"weighted ROC_AUC\"], hover_data=[\"overall_time\"], barmode=\"group\")\n",
    "py.iplot(fig)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
